{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "JPVBc5ndpFIX"
   },
   "source": [
    "# TextAttack & AllenNLP \n",
    "\n",
    "This is an example of testing adversarial attacks from TextAttack on pretrained models provided by AllenNLP. \n",
    "\n",
    "In a few lines of code, we load a sentiment analysis model trained on the Stanford Sentiment Treebank and configure it with a TextAttack model wrapper. Then, we initialize the TextBugger attack and run the attack on a few samples from the SST-2 train set.\n",
    "\n",
    "For more information on AllenNLP pre-trained models: https://docs.allennlp.org/models/main/\n",
    "\n",
    "For more information about the TextBugger attack: https://arxiv.org/abs/1812.05271"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "AyPMGcz0qLfK"
   },
   "source": [
    "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/QData/TextAttack/blob/master/docs/2notebook/Example_2_allennlp.ipynb)\n",
    "\n",
    "[![View Source on GitHub](https://img.shields.io/badge/github-view%20source-black.svg)](https://github.com/QData/TextAttack/blob/master/docs/2notebook/Example_2_allennlp.ipynb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip install allennlp allennlp_models > /dev/null"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "_br6Xvsif9SA",
    "outputId": "224cc851-0e9d-4454-931c-64bd3b7af400"
   },
   "outputs": [],
   "source": [
    "from allennlp.predictors import Predictor\n",
    "import allennlp_models.classification\n",
    "\n",
    "import textattack\n",
    "\n",
    "class AllenNLPModel(textattack.models.wrappers.ModelWrapper):\n",
    "    def __init__(self):\n",
    "        self.predictor = Predictor.from_path(\"https://storage.googleapis.com/allennlp-public-models/basic_stanford_sentiment_treebank-2020.06.09.tar.gz\")\n",
    "        self.model = self.predictor._model\n",
    "        self.tokenizer = self.predictor._dataset_reader._tokenizer\n",
    "\n",
    "    def __call__(self, text_input_list):\n",
    "        outputs = []\n",
    "        for text_input in text_input_list:\n",
    "            outputs.append(self.model.predict(sentence=text_input))\n",
    "        # For each output, outputs['logits'] contains the logits where\n",
    "        # index 0 corresponds to the positive and index 1 corresponds \n",
    "        # to the negative score. We reverse the outputs (by reverse slicing,\n",
    "        # [::-1]) so that negative comes first and positive comes second.\n",
    "        return [output['logits'][::-1] for output in outputs]\n",
    "\n",
    "model_wrapper = AllenNLPModel()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "MDRWI5Psb85g",
    "outputId": "db7f8f94-0d78-45ea-a7ac-e12167c28365"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Reusing dataset glue (/p/qdata/jy2ma/.cache/textattack/datasets/glue/sst2/1.0.0/dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad)\n",
      "textattack: Loading \u001b[94mdatasets\u001b[0m dataset \u001b[94mglue\u001b[0m, subset \u001b[94msst2\u001b[0m, split \u001b[94mtrain\u001b[0m.\n",
      "textattack: Unknown if model of class <class 'allennlp.predictors.text_classifier.TextClassifierPredictor'> compatible with goal function <class 'textattack.goal_functions.classification.untargeted_classification.UntargetedClassification'>.\n",
      "  0%|          | 0/10 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Attack(\n",
      "  (search_method): GreedyWordSwapWIR(\n",
      "    (wir_method):  delete\n",
      "  )\n",
      "  (goal_function):  UntargetedClassification\n",
      "  (transformation):  CompositeTransformation(\n",
      "    (0): WordSwapRandomCharacterInsertion(\n",
      "        (random_one):  True\n",
      "      )\n",
      "    (1): WordSwapRandomCharacterDeletion(\n",
      "        (random_one):  True\n",
      "      )\n",
      "    (2): WordSwapNeighboringCharacterSwap(\n",
      "        (random_one):  True\n",
      "      )\n",
      "    (3): WordSwapHomoglyphSwap\n",
      "    (4): WordSwapEmbedding(\n",
      "        (max_candidates):  5\n",
      "        (embedding):  WordEmbedding\n",
      "      )\n",
      "    )\n",
      "  (constraints): \n",
      "    (0): UniversalSentenceEncoder(\n",
      "        (metric):  angular\n",
      "        (threshold):  0.8\n",
      "        (window_size):  inf\n",
      "        (skip_text_shorter_than_window):  False\n",
      "        (compare_against_original):  True\n",
      "      )\n",
      "    (1): RepeatModification\n",
      "    (2): StopwordModification\n",
      "  (is_black_box):  True\n",
      ") \n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using /p/qdata/jy2ma/.cache/textattack to cache modules.\n",
      "[Succeeded / Failed / Skipped / Total] 1 / 1 / 0 / 2:  20%|██        | 2/10 [00:06<00:27,  3.46s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------------------------------------- Result 1 ---------------------------------------------\n",
      "\u001b[91mNegative (95%)\u001b[0m --> \u001b[92mPositive (93%)\u001b[0m\n",
      "\n",
      "\u001b[91mhide\u001b[0m new secretions from the parental units \n",
      "\n",
      "\u001b[92mconcealing\u001b[0m new secretions from the parental units \n",
      "\n",
      "\n",
      "--------------------------------------------- Result 2 ---------------------------------------------\n",
      "\u001b[91mNegative (96%)\u001b[0m --> \u001b[91m[FAILED]\u001b[0m\n",
      "\n",
      "contains no wit , only labored gags \n",
      "\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Succeeded / Failed / Skipped / Total] 1 / 2 / 1 / 4:  40%|████      | 4/10 [00:07<00:10,  1.80s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------------------------------------- Result 3 ---------------------------------------------\n",
      "\u001b[92mPositive (100%)\u001b[0m --> \u001b[91m[FAILED]\u001b[0m\n",
      "\n",
      "that loves its characters and communicates something rather beautiful about human nature \n",
      "\n",
      "\n",
      "--------------------------------------------- Result 4 ---------------------------------------------\n",
      "\u001b[92mPositive (82%)\u001b[0m --> \u001b[37m[SKIPPED]\u001b[0m\n",
      "\n",
      "remains utterly satisfied to remain the same throughout \n",
      "\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Succeeded / Failed / Skipped / Total] 2 / 2 / 1 / 5:  50%|█████     | 5/10 [00:07<00:07,  1.52s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------------------------------------- Result 5 ---------------------------------------------\n",
      "\u001b[91mNegative (98%)\u001b[0m --> \u001b[92mPositive (52%)\u001b[0m\n",
      "\n",
      "on the \u001b[91mworst\u001b[0m \u001b[91mrevenge-of-the-nerds\u001b[0m clichés the filmmakers could \u001b[91mdredge\u001b[0m up \n",
      "\n",
      "on the \u001b[92mpire\u001b[0m \u001b[92mreveng-of-the-nerds\u001b[0m clichés the filmmakers could \u001b[92mdragging\u001b[0m up \n",
      "\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Succeeded / Failed / Skipped / Total] 2 / 3 / 1 / 6:  60%|██████    | 6/10 [00:07<00:05,  1.32s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------------------------------------- Result 6 ---------------------------------------------\n",
      "\u001b[91mNegative (99%)\u001b[0m --> \u001b[91m[FAILED]\u001b[0m\n",
      "\n",
      "that 's far too tragic to merit such superficial treatment \n",
      "\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Succeeded / Failed / Skipped / Total] 3 / 4 / 1 / 8:  80%|████████  | 8/10 [00:09<00:02,  1.13s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------------------------------------- Result 7 ---------------------------------------------\n",
      "\u001b[92mPositive (98%)\u001b[0m --> \u001b[91mNegative (62%)\u001b[0m\n",
      "\n",
      "\u001b[92mdemonstrates\u001b[0m that the \u001b[92mdirector\u001b[0m of such \u001b[92mhollywood\u001b[0m blockbusters as patriot games can still \u001b[92mturn\u001b[0m out a \u001b[92msmall\u001b[0m , personal \u001b[92mfilm\u001b[0m with an emotional \u001b[92mwallop\u001b[0m . \n",
      "\n",
      "\u001b[91mshows\u001b[0m that the \u001b[91mdirectors\u001b[0m of such \u001b[91mtinseltown\u001b[0m blockbusters as patriot games can still \u001b[91mturning\u001b[0m out a \u001b[91mtiny\u001b[0m , personal \u001b[91mmovies\u001b[0m with an emotional \u001b[91mbatting\u001b[0m . \n",
      "\n",
      "\n",
      "--------------------------------------------- Result 8 ---------------------------------------------\n",
      "\u001b[92mPositive (90%)\u001b[0m --> \u001b[91m[FAILED]\u001b[0m\n",
      "\n",
      "of saucy \n",
      "\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Succeeded / Failed / Skipped / Total] 4 / 5 / 1 / 10: 100%|██████████| 10/10 [00:09<00:00,  1.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------------------------------------- Result 9 ---------------------------------------------\n",
      "\u001b[91mNegative (99%)\u001b[0m --> \u001b[91m[FAILED]\u001b[0m\n",
      "\n",
      "a depressed fifteen-year-old 's suicidal poetry \n",
      "\n",
      "\n",
      "--------------------------------------------- Result 10 ---------------------------------------------\n",
      "\u001b[92mPositive (79%)\u001b[0m --> \u001b[91mNegative (65%)\u001b[0m\n",
      "\n",
      "are more \u001b[92mdeeply\u001b[0m thought through than in most ` right-thinking ' films \n",
      "\n",
      "are more \u001b[91mseriously\u001b[0m thought through than in most ` right-thinking ' films \n",
      "\n",
      "\n",
      "\n",
      "+-------------------------------+--------+\n",
      "| Attack Results                |        |\n",
      "+-------------------------------+--------+\n",
      "| Number of successful attacks: | 4      |\n",
      "| Number of failed attacks:     | 5      |\n",
      "| Number of skipped attacks:    | 1      |\n",
      "| Original accuracy:            | 90.0%  |\n",
      "| Accuracy under attack:        | 50.0%  |\n",
      "| Attack success rate:          | 44.44% |\n",
      "| Average perturbed word %:     | 20.95% |\n",
      "| Average num. words per input: | 9.5    |\n",
      "| Avg num queries:              | 34.67  |\n",
      "+-------------------------------+--------+\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[<textattack.attack_results.successful_attack_result.SuccessfulAttackResult at 0x7fb68d0028b0>,\n",
       " <textattack.attack_results.failed_attack_result.FailedAttackResult at 0x7fb685f0dbb0>,\n",
       " <textattack.attack_results.failed_attack_result.FailedAttackResult at 0x7fb689188040>,\n",
       " <textattack.attack_results.skipped_attack_result.SkippedAttackResult at 0x7fb695031250>,\n",
       " <textattack.attack_results.successful_attack_result.SuccessfulAttackResult at 0x7fb695031760>,\n",
       " <textattack.attack_results.failed_attack_result.FailedAttackResult at 0x7fb694b7abb0>,\n",
       " <textattack.attack_results.successful_attack_result.SuccessfulAttackResult at 0x7fb67cd36df0>,\n",
       " <textattack.attack_results.failed_attack_result.FailedAttackResult at 0x7fb694b7a880>,\n",
       " <textattack.attack_results.failed_attack_result.FailedAttackResult at 0x7fb694b7a790>,\n",
       " <textattack.attack_results.successful_attack_result.SuccessfulAttackResult at 0x7fb689ab1be0>]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from textattack.datasets import HuggingFaceDataset\n",
    "from textattack.attack_recipes import TextBuggerLi2018\n",
    "from textattack.attacker import Attacker\n",
    "\n",
    "\n",
    "dataset = HuggingFaceDataset(\"glue\", \"sst2\", \"train\")\n",
    "attack = TextBuggerLi2018.build(model_wrapper)\n",
    "\n",
    "attacker = Attacker(attack, dataset)\n",
    "attacker.attack_dataset()"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "[TextAttack] Model Example: AllenNLP",
   "provenance": []
  },
  "kernelspec": {
   "name": "python379jvsc74a57bd00aa23297d40f12761ebb1c384bf2965d5ecbdef2f9c005ee7346b9ec0bcc5588",
   "display_name": "Python 3.7.9 64-bit ('pytorch-gpu': pyenv)"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
